{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/tensorflow/mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/tensorflow/mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "test learnreat=0.5 result= 0.9036\n",
      "test learnreat=0.45 result= 0.9571\n",
      "test learnreat=0.405 result= 0.9641\n",
      "test learnreat=0.36450000000000005 result= 0.9724\n",
      "test learnreat=0.32805 result= 0.977\n",
      "test learnreat=0.29524500000000004 result= 0.9806\n",
      "test learnreat=0.2657205 result= 0.9863\n",
      "test learnreat=0.23914845000000004 result= 0.9832\n",
      "test learnreat=0.21523360500000005 result= 0.9837\n",
      "test learnreat=0.19371024450000005 result= 0.9865\n",
      "test learnreat=0.17433922005000005 result= 0.9865\n",
      "test learnreat=0.15690529804500003 result= 0.9885\n",
      "test learnreat=0.14121476824050005 result= 0.9872\n",
      "test learnreat=0.12709329141645004 result= 0.9887\n",
      "test learnreat=0.11438396227480505 result= 0.9888\n",
      "test learnreat=0.10294556604732454 result= 0.9889\n",
      "test learnreat=0.09265100944259208 result= 0.9874\n",
      "test learnreat=0.08338590849833288 result= 0.9888\n",
      "test learnreat=0.07504731764849959 result= 0.9897\n",
      "test learnreat=0.06754258588364964 result= 0.9888\n",
      "test learnreat=0.060788327295284675 result= 0.9894\n",
      "test learnreat=0.05470949456575621 result= 0.9899\n",
      "test learnreat=0.04923854510918059 result= 0.9891\n",
      "test learnreat=0.044314690598262534 result= 0.99\n",
      "test learnreat=0.03988322153843628 result= 0.9894\n",
      "test learnreat=0.03589489938459265 result= 0.9888\n",
      "test learnreat=0.03230540944613339 result= 0.9904\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Import data\n",
    "data_dir = '/tmp/tensorflow/mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "# Train\n",
    "examNum = mnist.train.num_examples // 100\n",
    "maxstep =80\n",
    "for step in range(maxstep):\n",
    "        \n",
    "    lr = 0.5 * (0.9 ** step)\n",
    "        \n",
    "    for i in range(examNum):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run([train_step, cross_entropy, l2_loss, total_loss],\n",
    "                 feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "\n",
    "\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32))\n",
    "    total = 0\n",
    "    for i in range(40):#测试数据共1w条\n",
    "        accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32))\n",
    "        testX, testY = mnist.test.next_batch(250)\n",
    "        total += sess.run(accuracy, feed_dict={x: testX,\n",
    "                                    y_: testY, keep_prob:1})\n",
    "    print(\"test learnreat={0} result= {1}\".format(lr, total/10000))\n",
    "    if total > 9900:\n",
    "        break\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "最后做下总结:\n",
    "学习率很重要,一开始学习率需要足够大,快速学习,等到 后面学习率需要变小,上面的学习率 看来还是设置得不是很好, 后面的学习率应该降得更快一点\n",
    "\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
